{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 推荐系统的数据预计算"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<module 'sys' (built-in)>"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import sys\n",
    "from imp import reload\n",
    "reload(sys)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import scipy.sparse as ss\n",
    "import _pickle as cPickle\n",
    "import scipy.io as sio\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "from collections import defaultdict"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 读取数据\n",
    "用户(10万)、歌曲（3万）、播放次数、歌曲元数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = pd.read_csv('triplet_dataset_sub_song_merged.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user</th>\n",
       "      <th>song</th>\n",
       "      <th>listen_count</th>\n",
       "      <th>title</th>\n",
       "      <th>release</th>\n",
       "      <th>artist_name</th>\n",
       "      <th>year</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>5a905f000fc1ff3df7ca807d57edb608863db05d</td>\n",
       "      <td>SOAFTRR12AF72A8D4D</td>\n",
       "      <td>1</td>\n",
       "      <td>Harder Better Faster Stronger</td>\n",
       "      <td>Discovery</td>\n",
       "      <td>Daft Punk</td>\n",
       "      <td>2007</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>5a905f000fc1ff3df7ca807d57edb608863db05d</td>\n",
       "      <td>SOAIILB12A58A776F7</td>\n",
       "      <td>3</td>\n",
       "      <td>Phantom Part 1.5 (Album Version)</td>\n",
       "      <td>A Cross The Universe</td>\n",
       "      <td>Justice</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>5a905f000fc1ff3df7ca807d57edb608863db05d</td>\n",
       "      <td>SOAJJDS12A8C13A3FB</td>\n",
       "      <td>1</td>\n",
       "      <td>I Got Mine</td>\n",
       "      <td>Attack &amp; Release</td>\n",
       "      <td>The Black Keys</td>\n",
       "      <td>2008</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>5a905f000fc1ff3df7ca807d57edb608863db05d</td>\n",
       "      <td>SOAMDXO12A8C131E2F</td>\n",
       "      <td>2</td>\n",
       "      <td>Pogo</td>\n",
       "      <td>Idealism</td>\n",
       "      <td>Digitalism</td>\n",
       "      <td>2007</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5a905f000fc1ff3df7ca807d57edb608863db05d</td>\n",
       "      <td>SOAMPRJ12A8AE45F38</td>\n",
       "      <td>20</td>\n",
       "      <td>Rorol</td>\n",
       "      <td>Identification Parade</td>\n",
       "      <td>Octopus Project</td>\n",
       "      <td>2002</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       user                song  listen_count  \\\n",
       "0  5a905f000fc1ff3df7ca807d57edb608863db05d  SOAFTRR12AF72A8D4D             1   \n",
       "1  5a905f000fc1ff3df7ca807d57edb608863db05d  SOAIILB12A58A776F7             3   \n",
       "2  5a905f000fc1ff3df7ca807d57edb608863db05d  SOAJJDS12A8C13A3FB             1   \n",
       "3  5a905f000fc1ff3df7ca807d57edb608863db05d  SOAMDXO12A8C131E2F             2   \n",
       "4  5a905f000fc1ff3df7ca807d57edb608863db05d  SOAMPRJ12A8AE45F38            20   \n",
       "\n",
       "                              title                release      artist_name  \\\n",
       "0     Harder Better Faster Stronger              Discovery        Daft Punk   \n",
       "1  Phantom Part 1.5 (Album Version)   A Cross The Universe          Justice   \n",
       "2                        I Got Mine       Attack & Release   The Black Keys   \n",
       "3                              Pogo               Idealism       Digitalism   \n",
       "4                             Rorol  Identification Parade  Octopus Project   \n",
       "\n",
       "   year  \n",
       "0  2007  \n",
       "1     0  \n",
       "2  2008  \n",
       "3  2007  \n",
       "4  2002  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 493950 entries, 0 to 493949\n",
      "Data columns (total 7 columns):\n",
      "user            493950 non-null object\n",
      "song            493950 non-null object\n",
      "listen_count    493950 non-null int64\n",
      "title           493950 non-null object\n",
      "release         493950 non-null object\n",
      "artist_name     493950 non-null object\n",
      "year            493950 non-null int64\n",
      "dtypes: int64(2), object(5)\n",
      "memory usage: 26.4+ MB\n"
     ]
    }
   ],
   "source": [
    "data.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 子集抽取\n",
    "要计算用户的item与所有3万首歌的相似度太慢，可考虑只推荐最流行的500首歌"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "song_count_df = pd.read_csv('song_playcount_df.csv')\n",
    "song_count_sub_df = song_count_df.head(n = 500)\n",
    "top5h = list(song_count_sub_df.song)\n",
    "data_sub = data[data.song.isin(top5h)]\n",
    "# del data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(146566, 7)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_sub.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 对抽取过的子集切分训练数据和测试数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data_sub, test_data_sub = train_test_split(data_sub, test_size=0.4,random_state=6)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 隐式打分\n",
    "由于这个数据集中并没有用户对物品的显式打分，需要将播放次数转换为分数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_sub_total_count_df = train_data_sub[['user','listen_count']].groupby('user').sum()\n",
    "data_sub_total_count_df.rename(columns={'listen_count':'total_count'}, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data_sub = pd.merge(train_data_sub, data_sub_total_count_df, on='user')\n",
    "del data_sub_total_count_df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 将用户对某歌曲播放的次数占所有歌曲播放次数总和的比例作为打分"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data_sub['fractional_play_count'] = train_data_sub['listen_count']/train_data_sub['total_count']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user</th>\n",
       "      <th>song</th>\n",
       "      <th>listen_count</th>\n",
       "      <th>title</th>\n",
       "      <th>release</th>\n",
       "      <th>artist_name</th>\n",
       "      <th>year</th>\n",
       "      <th>total_count</th>\n",
       "      <th>fractional_play_count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>324f45310e64bbf8775717135e35e99811a8e937</td>\n",
       "      <td>SOOXLIG12AB0185803</td>\n",
       "      <td>21</td>\n",
       "      <td>Spell</td>\n",
       "      <td>Abattoir Blues/The Lyre of Orpheus</td>\n",
       "      <td>Nick Cave &amp; The Bad Seeds</td>\n",
       "      <td>2004</td>\n",
       "      <td>424</td>\n",
       "      <td>0.049528</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>324f45310e64bbf8775717135e35e99811a8e937</td>\n",
       "      <td>SOAKMDU12A8C1346A9</td>\n",
       "      <td>1</td>\n",
       "      <td>Such Great Heights</td>\n",
       "      <td>Grey's Anatomy Original Soundtrack</td>\n",
       "      <td>The Postal Service</td>\n",
       "      <td>2003</td>\n",
       "      <td>424</td>\n",
       "      <td>0.002358</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>324f45310e64bbf8775717135e35e99811a8e937</td>\n",
       "      <td>SOTVLQY12A58A798C2</td>\n",
       "      <td>3</td>\n",
       "      <td>Home</td>\n",
       "      <td>Up From Below</td>\n",
       "      <td>Edward Sharpe &amp; The Magnetic Zeros</td>\n",
       "      <td>2009</td>\n",
       "      <td>424</td>\n",
       "      <td>0.007075</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>324f45310e64bbf8775717135e35e99811a8e937</td>\n",
       "      <td>SOANQFY12AB0183239</td>\n",
       "      <td>2</td>\n",
       "      <td>Uprising</td>\n",
       "      <td>Uprising</td>\n",
       "      <td>Muse</td>\n",
       "      <td>0</td>\n",
       "      <td>424</td>\n",
       "      <td>0.004717</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>324f45310e64bbf8775717135e35e99811a8e937</td>\n",
       "      <td>SOOJJCT12A6310E1C0</td>\n",
       "      <td>3</td>\n",
       "      <td>Here Without You</td>\n",
       "      <td>Here Without You</td>\n",
       "      <td>3 Doors Down</td>\n",
       "      <td>2002</td>\n",
       "      <td>424</td>\n",
       "      <td>0.007075</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       user                song  listen_count  \\\n",
       "0  324f45310e64bbf8775717135e35e99811a8e937  SOOXLIG12AB0185803            21   \n",
       "1  324f45310e64bbf8775717135e35e99811a8e937  SOAKMDU12A8C1346A9             1   \n",
       "2  324f45310e64bbf8775717135e35e99811a8e937  SOTVLQY12A58A798C2             3   \n",
       "3  324f45310e64bbf8775717135e35e99811a8e937  SOANQFY12AB0183239             2   \n",
       "4  324f45310e64bbf8775717135e35e99811a8e937  SOOJJCT12A6310E1C0             3   \n",
       "\n",
       "                title                             release  \\\n",
       "0               Spell  Abattoir Blues/The Lyre of Orpheus   \n",
       "1  Such Great Heights  Grey's Anatomy Original Soundtrack   \n",
       "2                Home                       Up From Below   \n",
       "3            Uprising                            Uprising   \n",
       "4    Here Without You                    Here Without You   \n",
       "\n",
       "                          artist_name  year  total_count  \\\n",
       "0           Nick Cave & The Bad Seeds  2004          424   \n",
       "1                  The Postal Service  2003          424   \n",
       "2  Edward Sharpe & The Magnetic Zeros  2009          424   \n",
       "3                                Muse     0          424   \n",
       "4                        3 Doors Down  2002          424   \n",
       "\n",
       "   fractional_play_count  \n",
       "0               0.049528  \n",
       "1               0.002358  \n",
       "2               0.007075  \n",
       "3               0.004717  \n",
       "4               0.007075  "
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_data_sub.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "songs = top5h\n",
    "users = data_sub.user.unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "song_len:500\n",
      "user_len:4924\n"
     ]
    }
   ],
   "source": [
    "song_len = len(songs)\n",
    "user_len = len(users)\n",
    "print(\"song_len:%s\"%song_len)\n",
    "print(\"user_len:%s\"%user_len)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 统计每个用户听过哪些歌、每首歌被谁听过\n",
    "user_songs = defaultdict(set)\n",
    "song_users = defaultdict(set)\n",
    "# 评分矩阵\n",
    "user_song_scores = ss.dok_matrix((user_len,song_len))\n",
    "# 建立测试数据的用户-歌曲关系矩阵\n",
    "user_song_scores_test = ss.dok_matrix((user_len,song_len))\n",
    "#建立倒排索引\n",
    "song_index_dict = {}\n",
    "user_index_dict = {}\n",
    "\n",
    "for i,u in enumerate(songs):\n",
    "    song_index_dict[u] = i\n",
    "    \n",
    "for i,u in enumerate(users):\n",
    "    user_index_dict[u] = i \n",
    "\n",
    "for index,row in test_data_sub.iterrows():\n",
    "    user_song_scores_test[user_index_dict[row.user],song_index_dict[row.song]] = 1\n",
    "\n",
    "for index,row in train_data_sub.iterrows():\n",
    "    user_index = user_index_dict[row.user]\n",
    "    song_index = song_index_dict[row.song]\n",
    "    user_songs[user_index].add(song_index)\n",
    "    song_users[song_index].add(user_index)\n",
    "    user_song_scores[user_index,song_index] = row.fractional_play_count\n",
    "#     UserSongScores[song_index,user_index] = row.fractional_play_count\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 保存倒排表\n",
    "cPickle.dump(user_songs, open('user_songs.pkl','wb'))\n",
    "cPickle.dump(song_users, open('song_users.pkl','wb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 保存倒排索引\n",
    "cPickle.dump(song_index_dict, open('song_index.pkl','wb'))\n",
    "cPickle.dump(user_index_dict, open('user_index.pkl','wb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 保存评分矩阵\n",
    "sio.mmwrite('user_song_scores', user_song_scores)\n",
    "# 保存测试数据的用户-歌曲关系矩阵\n",
    "cPickle.dump(user_song_scores_test, open('user_song_scores_test.pkl','wb'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 计算相似度"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 用户播放过歌曲就表示为1，否则为0（二值化），这样物品之间的相似度为播放两个歌曲的用户交集除以播放两个歌曲的用户并集\n",
    "song_sim_matrix = ss.dok_matrix((song_len, song_len))\n",
    "for i in range(song_len):\n",
    "    song_sim_matrix[i,i] = 1\n",
    "    for j in range(i+1,song_len):\n",
    "        # i的用户\n",
    "        users_i_set = song_users[i]\n",
    "        # j的用户\n",
    "        users_j_set = song_users[j]\n",
    "        # 取交集\n",
    "        intersection = users_i_set & users_j_set\n",
    "        inter_len = len(intersection)\n",
    "        if inter_len == 0:\n",
    "            song_sim_matrix[i,j] = 0\n",
    "        else:\n",
    "            # 取并集\n",
    "            union = users_i_set | users_j_set\n",
    "            union_len = len(union)\n",
    "            song_sim_matrix[i,j] = float(inter_len)/float(union_len)\n",
    "        song_sim_matrix[j,i] = song_sim_matrix[i,j]\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "cPickle.dump(song_sim_matrix, open('song_sim_matrix.pkl','wb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 类似的，两个用户之间的相似度可用两个用户播放歌曲的交集除以两个用户播放歌曲的并集表示\n",
    "user_sim_matrix = ss.dok_matrix((user_len, user_len))\n",
    "for i in range(user_len):\n",
    "    user_sim_matrix[i,i] = 1\n",
    "    for j in range(i+1,user_len):\n",
    "        # i的歌曲\n",
    "        songs_i_set = user_songs[i]\n",
    "        # j的歌曲\n",
    "        songs_j_set = user_songs[j]\n",
    "        # 取交集\n",
    "        intersection = songs_i_set & songs_j_set\n",
    "        inter_len = len(intersection)\n",
    "        if inter_len == 0:\n",
    "            user_sim_matrix[i,j] = 0\n",
    "        else:\n",
    "            # 取并集\n",
    "            union = songs_i_set | songs_j_set\n",
    "            union_len = len(union)\n",
    "            user_sim_matrix[i,j] = float(inter_len)/float(union_len)\n",
    "        user_sim_matrix[j,i] = user_sim_matrix[i,j]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "cPickle.dump(user_sim_matrix, open('user_sim_matrix.pkl','wb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
